package com.kdtech.utils;

import java.io.IOException;

import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.kd.crawler.parser.url.UrlParser;


/**
 *
 * 去除网页中的推荐链接，导航，广告页
 * @author KK
 *
 */
public class HtmlCleaner {

	public static final int MIN_WIDTH = 200;
	public static final int MIN_HEIGHT = 200;
	public static String getContentHtml(String url,Elements element) {
		if (element==null ){
			return null;
		}
		Elements elems = element.clone();
		StringBuffer sb = new StringBuffer();
		for (Element e : elems) {
			String contentHtml = getContentHtml(url, e);
			if(contentHtml!=null){
				sb.append(contentHtml);
			}
		}
		String html = sb.toString();
		html = html.replaceAll("[ ]+[\r\n]", " ");
		html = html.replace("</p>", "</p>\r\n");
		return html;
	}

	public static String getContentHtml(String url,Element doc) {
		if (doc==null || (StringUtils.isBlank(doc.text()) && doc.select("img").size()==0) ){
			return null;
		}
		cleanNode(doc);
		Elements children = doc.children();
		for (Element element : children) {
			cleanNode(element);
		}

//		Elements select3 = doc.select("script");
//		System.out.println(select3.html());
		//清除输入框，及textarea区域


		//清除广告
		/*Elements select = doc.select("a,img");
		for (Element element : select) {
			Element e = findParent(element);
			if (e==null || e.parent()==null){
				continue;
			}
			String text = e.text();
			if(text==null || text.trim().equals("")){
				if (e.select("a").size()>=3){
					removeNode(e);
				}
			}
			String aText = e.select("a").text();
			if(text!=null && !text.equals("")){
				int textLength = getTextLength(text);
				int atextLlength = getTextLength(aText);
				if (textLength < 100){
					double rate =(double)textLength/atextLlength;
					if(rate > 0.5){
//						System.out.println(getTextLength(aText) +"->" + getTextLength(text)  + "->" + text.length() + "->" + text);
						removeNode(e);
					}else{
//					if (e.attr("class").equals("txtcenter bt3 f12_575656")){
//						System.out.println(rate + "->" + e.nodeName() + "." +e.attr("class"));
//					}
					}
				}
			}
		}
*/
//		Elements uls = doc.select("ul");
//		for (Element e : uls) {
//			removeNode(e);
//		}

//		Elements inputs = doc.select("input,select,textarea,form,button");
//		for (Element e : inputs) {
//			removeNode(e);
//		}

		//清除小图片
		Elements imgs = doc.select("img");
		for (Element img : imgs) {
			String width = img.attr("width").trim();
			if (width!=null  && !width.equals("")){
				try {
						if (width.contains("px")){
							width = width.toLowerCase().replace("px", "").replace(" ", "");
							if (NumberUtils.matchNumber(width) < MIN_WIDTH){
								removeNode(img);
								continue;
							}
						}
				} catch (NumberFormatException e) {
					e.printStackTrace();
				}
			}
			String height = img.attr("height");
			if (height!=null && !height.equals("")){
				try {
						if (height.contains("px")){
							height = height.toLowerCase().replace("px", "").replace(" ", "");
							if (NumberUtils.matchNumber(height) < MIN_HEIGHT){
								removeNode(img);
								continue;
							}
						}
				} catch (NumberFormatException e) {
					e.printStackTrace();
				}
			}
		}



		//清除剩余的,"推荐","热点","喜欢","排行","跟贴","焦点","视频","资讯","信息","精选"信息
		removeHots(doc);

		Elements areas = null;

		//最后将无文字的块清除
		Elements select2 = doc.select("div,table,td,tr,span");
		for (Element element : select2) {
			if (getTextLength(element.text())==0 && element.select("img").size()==0){
				removeNode(element);
			}
		}

		//补全链接
		imgs = doc.select("[src]");
		for (Element img : imgs) {
			String attr = img.attr("src");
			if (notEmpty(attr)){
//				attr=CrawlHTML.complementURL(attr, url, doc, true);
				attr = UrlParser.completeURL(attr, url);
				img.attr("src",attr);
			}
		}
		imgs = doc.select("[href]");
		for (Element img : imgs) {
			String attr = img.attr("href");
			if (notEmpty(attr)){
//				attr=CrawlHTML.complementURL(attr, url, doc, true);
				attr = UrlParser.completeURL(attr, url);
				img.attr("href",attr);
			}
		}
		String html = doc.html();
		if (doc.nodeName() .equals("p")){
			html = doc.outerHtml();
		}
		html = removeTag(html,"table","tbody","tr","td","font");
		html = html.replaceAll("(?i)width[ ]*:[ ]*[0-9]+[ ]*[;]?", "");
		html = html.replaceAll("&nbsp;", " ");
		html = html.replaceAll("[|]", " ");

		//去除<!-- -->注释里面的内容
		while(true){
			int indexOf = html.indexOf("<!--");
			if (indexOf!=-1){
				int indexOf2 = html.indexOf("-->",indexOf);
				if (indexOf2!=-1){
					html = html.substring(0,indexOf) + html.substring(indexOf2+3);
				}else{
					break;
				}
			}else{
				break;
			}
		}
		html = html.trim();

		return html;
	}

	private static String removeTag(String html, String... tags) {
		for (String tag : tags) {
			html = html.replaceAll("<"+tag+"[^>]*>","");
			html = html.replaceAll("</"+tag+">","");
		}
		return html;
	}

	private static void cleanNode(Element e) {
//		if (getTextLength(e.text())<=1){
//			e.remove();
//			return ;
//		}
		String nodeName = e.nodeName();
		if (StringUtils.equalsIgnoreCaseAny(nodeName, "link","style","script","noscript","iframe","input","select","textarea","form","button")){
			e.remove();
			return ;
		}

		if (!StringUtils.equalsIgnoreCaseAny(nodeName, "p","br")){
			if(StringUtils.isBlank(e.text()) && e.select("img").size()==0) {
				e.remove();
				return;
			}
		}
		String style = e.attr("style");
		if (style!=null && style.contains("display:none")){
			e.remove();
			return ;
		}
		//移除事件 onclick,onload...
		Attributes attributes = e.attributes();
		for (Attribute a : attributes) {
			String key = a.getKey().toLowerCase();
			if(!StringUtils.equalsAny(key, "href","src","alt")){
				e.removeAttr(a.getKey());
			}
		}
//		e.removeAttr("id");
//		e.removeAttr("name");
//		e.removeAttr("class");
//		e.removeAttr("style");
//		e.removeAttr("title");
//		e.removeAttr(".");
//		e.removeAttr("itemscope");
//		e.removeAttr("itemtype");
		Elements children = e.children();
		for (Element element : children) {
			cleanNode(element);
		}

	}

	private static int getNoLinkText(Element parent) {
		return getTextLength(parent.text()) - getTextLength(parent.select("a,img").text());
	}

	private static int getTextLength(String text) {
		if (text==null) return 0;
		text = text.replace(">", "");
		text = text.replace("-", "");
		text = text.replace("|", "");
		text = text.replace("|", "");
		text = text.replaceAll("[\\s\\t\\n\\x0B\\f\\r  　]", "");
		return text.length();
	}

	private static boolean notEmpty(String attr) {
		return attr!=null && attr.trim().length()>0;
	}

	private static void removeHots(Element doc) {
		removeHots(doc, "h1","h2","h3","h4","h5","a");
	}

	private static void removeHots(Element doc,String... nodenames) {
		for (String nodename : nodenames) {
			Elements select = doc.select(nodename+":matchesOwn([\u4e00-\u9fa5]{2,6})");
			for (Element element : select) {
				int nodeTxtLength = getTextLength(element.text());
				if ( nodeTxtLength> 10){
					continue;
				}else{
					Element parentByNodeName = getParentByNodeName(element, "div","td","ul");
					if (parentByNodeName!=null){
						int parentTxtLength = getTextLength(parentByNodeName.text());
						if (parentTxtLength < 20 ){
							removeNode(parentByNodeName);
						}
					}
				}
			}
		}
	}

	private static void removeNode(Element node) {
		while(true){
			if (node==null || node.parent() == null){
				break;
			}
			if (getTextLength(node.parent().text())==0 && node.parent().select("img").size()==0){
				node = node.parent();
			}else{
				node.remove();
				break;
			}
		}
	}

	private static Element getParentByNodeName(Element e, String... nodenames) {
		Element parent = e.parent();
		if (parent==null) return null;
		String nodeName = parent.nodeName();
		if (StringUtils.equalsIgnoreCaseAny(nodeName, nodenames)){
			return parent;
		}
		return getParentByNodeName(parent,nodenames);
	}

	public static void main(String[] args) throws IOException {
		String html = " </p> <p>"	;
		html = html.replaceAll("[ ]*[\r\n]+</p>", "</p>");
		System.out.println(html);
		html = html.replace("</p>", "</p>\r\n");
		System.out.println(html);
//		String html = "WIDTH: 1872px;[]";
//		html = html.replaceAll("(?i)width[ ]*:[ ]*[0-9]+px[ ]*[;]?", "");
//		System.out.println(html);
//		String url = "http://news.southcn.com/china/content/2014-11/10/content_111913008.htm";
//		Document doc = Jsoup.connect(url).get();
//		doc.setBaseUri(url);
////		System.out.println(doc.html());
//		String removeAds = getContentHtml(url,doc);
//		System.out.println(removeAds);
//		IOUtil.WriteStringContent("d:/test.html", removeAds, "gbk");
//		Map<String, Object> update = new HashMap<String, Object>();
//		update.put("M_cleanHtml", removeAds);
//		new IPublichotinfoImpl().update(2889, update);
//		System.out.println(removeAds(doc.html()));
	}

}

